<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"><title>大数据之Hadoop02_集群搭建 | Oct25-X</title><meta name="keywords" content="Hadoop"><meta name="author" content="Oct25-X"><meta name="copyright" content="Oct25-X"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="集群规划     hadoop102 hadoop103 hadoop104    HDFS NameNode ✔     HDFS DataNode ✔ ✔ ✔   HDFS SecondaryNameNode   ✔   YARN ResourceManager  ✔    YARN NodeManager ✔ ✔ ✔    JobHistoryServer   ✔   JDK 安装 卸载现有">
<meta property="og:type" content="article">
<meta property="og:title" content="大数据之Hadoop02_集群搭建">
<meta property="og:url" content="https://oct25-x.atomgit.net/oct25-xxxxx/2023/05/29/03%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop02_%E9%9B%86%E7%BE%A4%E6%90%AD%E5%BB%BA/index.html">
<meta property="og:site_name" content="Oct25-X">
<meta property="og:description" content="集群规划     hadoop102 hadoop103 hadoop104    HDFS NameNode ✔     HDFS DataNode ✔ ✔ ✔   HDFS SecondaryNameNode   ✔   YARN ResourceManager  ✔    YARN NodeManager ✔ ✔ ✔    JobHistoryServer   ✔   JDK 安装 卸载现有">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://oct25-x.atomgit.net/oct25-xxxxx/img/cover_img/ctt3.png">
<meta property="article:published_time" content="2023-05-29T02:15:36.000Z">
<meta property="article:modified_time" content="2024-02-17T14:13:09.670Z">
<meta property="article:author" content="Oct25-X">
<meta property="article:tag" content="Hadoop">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://oct25-x.atomgit.net/oct25-xxxxx/img/cover_img/ctt3.png"><link rel="shortcut icon" href="/oct25-xxxxx/img/logo.jpg"><link rel="canonical" href="https://oct25-x.atomgit.net/oct25-xxxxx/2023/05/29/03%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop02_%E9%9B%86%E7%BE%A4%E6%90%AD%E5%BB%BA/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/oct25-xxxxx/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@6/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = { 
  root: '/oct25-xxxxx/',
  algolia: undefined,
  localSearch: {"path":"search.json","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: {"defaultEncoding":2,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"簡"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":false},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: undefined,
  source: {
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isAnchor: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '大数据之Hadoop02_集群搭建',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2024-02-17 22:13:09'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const detectApple = () => {
      if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><link rel="stylesheet" href="/css/rightMenu.css"><meta name="generator" content="Hexo 6.1.0"></head><body><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/oct25-xxxxx/img/avatar.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data is-center"><div class="data-item"><a href="/oct25-xxxxx/archives/"><div class="headline">文章</div><div class="length-num">52</div></a></div><div class="data-item"><a href="/oct25-xxxxx/tags/"><div class="headline">标签</div><div class="length-num">17</div></a></div><div class="data-item"><a href="/oct25-xxxxx/categories/"><div class="headline">分类</div><div class="length-num">8</div></a></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('/oct25-xxxxx/img/top_img/top_bg2.jpg')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/oct25-xxxxx/">Oct25-X</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/oct25-xxxxx/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">大数据之Hadoop02_集群搭建</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2023-05-29T02:15:36.000Z" title="发表于 2023-05-29 10:15:36">2023-05-29</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2024-02-17T14:13:09.670Z" title="更新于 2024-02-17 22:13:09">2024-02-17</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/oct25-xxxxx/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE%E7%BB%84%E4%BB%B6/">大数据组件</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="大数据之Hadoop02_集群搭建"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="集群规划"><a href="#集群规划" class="headerlink" title="集群规划"></a>集群规划</h1><table>
<thead>
<tr>
<th></th>
<th></th>
<th>hadoop102</th>
<th>hadoop103</th>
<th>hadoop104</th>
</tr>
</thead>
<tbody><tr>
<td>HDFS</td>
<td>NameNode</td>
<td>✔</td>
<td></td>
<td></td>
</tr>
<tr>
<td>HDFS</td>
<td>DataNode</td>
<td>✔</td>
<td>✔</td>
<td>✔</td>
</tr>
<tr>
<td>HDFS</td>
<td>SecondaryNameNode</td>
<td></td>
<td></td>
<td>✔</td>
</tr>
<tr>
<td>YARN</td>
<td>ResourceManager</td>
<td></td>
<td>✔</td>
<td></td>
</tr>
<tr>
<td>YARN</td>
<td>NodeManager</td>
<td>✔</td>
<td>✔</td>
<td>✔</td>
</tr>
<tr>
<td></td>
<td>JobHistoryServer</td>
<td></td>
<td></td>
<td>✔</td>
</tr>
</tbody></table>
<h1 id="JDK-安装"><a href="#JDK-安装" class="headerlink" title="JDK 安装"></a>JDK 安装</h1><ol>
<li>卸载现有JDK</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">查看</span></span><br><span class="line">rpm -qa | grep java</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">卸载</span></span><br><span class="line">rpm -qa | grep java | xargs sudo rpm -e --nodeps</span><br></pre></td></tr></table></figure>

<ol start="2">
<li>上传，解压</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">tar -zxvf jdk-8u144-linux-x64.tar.gz -C /opt/module/</span><br></pre></td></tr></table></figure>

<ol start="3">
<li>配置环境变量</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">sudo vim /etc/profile.d/myEnv.sh</span><br></pre></td></tr></table></figure>

<p>添加以下代码内容：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_">#</span><span class="language-bash">JAVA_HOME</span></span><br><span class="line">export JAVA_HOME=/opt/module/jdk1.8.0_144</span><br><span class="line">export PATH=$PATH:$JAVA_HOME/bin</span><br></pre></td></tr></table></figure>

<ol start="4">
<li>集群间同步 JDK 和 &#x2F;etc&#x2F;profile.d&#x2F;myEnv.sh</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">xsync jdk1.8.0_144</span><br><span class="line">sudo xsync /etc/profile.d/myEnv.sh</span><br></pre></td></tr></table></figure>

<ol start="5">
<li>刷新环境变量</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">source /etc/profile</span><br></pre></td></tr></table></figure>

<ol start="6">
<li>测试</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">java -version</span><br><span class="line">openjdk version &quot;1.8.0_242&quot;</span><br><span class="line">OpenJDK Runtime Environment (build 1.8.0_242-b08)</span><br><span class="line">OpenJDK 64-Bit Server VM (build 25.242-b08, mixed mode)</span><br></pre></td></tr></table></figure>

<h1 id="集群间免密登录"><a href="#集群间免密登录" class="headerlink" title="集群间免密登录"></a>集群间免密登录</h1><ol>
<li>生成密钥</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">ssh-keygen -t rsa # 三次回车</span><br></pre></td></tr></table></figure>

<ol start="2">
<li>发送公钥到集群的其他节点</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">ssh-copy-id hadoop102</span><br></pre></td></tr></table></figure>

<p><strong>Tips</strong>：需要 hadoop102 和 hadoop103 与其他节点完成免密登录。</p>
<ol start="3">
<li>测试</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">ssh hadoopp103</span><br></pre></td></tr></table></figure>

<h1 id="Hadoop-集群搭建"><a href="#Hadoop-集群搭建" class="headerlink" title="Hadoop 集群搭建"></a>Hadoop 集群搭建</h1><h2 id="一、安装-Hadoop"><a href="#一、安装-Hadoop" class="headerlink" title="一、安装 Hadoop"></a>一、安装 Hadoop</h2><ol>
<li>上传，解压</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">tar -zxvf hadoop-3.1.3.tar.gz -C /opt/module/</span><br></pre></td></tr></table></figure>

<ol start="2">
<li>配置环境变量</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">sudo vim /etc/profile.d/myEnv.sh</span><br></pre></td></tr></table></figure>

<p>添加以下代码内容：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_">#</span><span class="language-bash">HADOOP_HOME</span></span><br><span class="line">export HADOOP_HOME=/opt/module/hadoop-3.1.3</span><br><span class="line">export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin</span><br></pre></td></tr></table></figure>

<ol start="3">
<li>刷新环境变量</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">source /etc/profile</span><br></pre></td></tr></table></figure>

<ol start="4">
<li>测试</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hadoop version</span><br></pre></td></tr></table></figure>

<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">Hadoop 3.1.3</span><br><span class="line">Source code repository https://gitbox.apache.org/repos/asf/hadoop.git -r ba631c436b806728f8ec2f54ab1e289526c90579</span><br><span class="line">Compiled by ztang on 2019-09-12T02:47Z</span><br><span class="line">Compiled with protoc 2.5.0</span><br><span class="line">From source with checksum ec785077c385118ac91aadde5ec9799</span><br><span class="line">This command was run using /opt/module/hadoop-3.1.3/share/hadoop/common/hadoop-common-3.1.3.jar</span><br></pre></td></tr></table></figure>

<p>控制台出现 Hadoop 版本信息，即为配置成功</p>
<h2 id="二、修改配置文件"><a href="#二、修改配置文件" class="headerlink" title="二、修改配置文件"></a>二、修改配置文件</h2><p>需要修改的配置文件如下：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">ore-site.xml	核心配置文件</span><br><span class="line">hdfs-site.xml	HDFS配置文件</span><br><span class="line">yarn-site.xml	Yarn配置文件</span><br><span class="line">mapred-site.xml	MapReduce配置文件</span><br><span class="line">workers			工作节点配置文件</span><br></pre></td></tr></table></figure>

<p>配置文件存储目录如下：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$HADOOP_HOME/etc/hadoop/</span><br></pre></td></tr></table></figure>

<h3 id="core-site-xml"><a href="#core-site-xml" class="headerlink" title="core-site.xml"></a>core-site.xml</h3><figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">&lt;!-- 指定HDFS中NameNode的地址 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>fs.defaultFS<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">value</span>&gt;</span>hdfs://hadoop102:8020<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- 指定Hadoop运行时产生文件的存储目录 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>hadoop.tmp.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">value</span>&gt;</span>/opt/module/hadoop-3.1.3/data/tmp<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- 配置HDFS网页登录使用的静态用户名为 xkee --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>hadoop.http.staticuser.user<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>xkee<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h3 id="hdfs-site-xml"><a href="#hdfs-site-xml" class="headerlink" title="hdfs-site.xml"></a>hdfs-site.xml</h3><figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">&lt;!-- namenode web访问地址 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.namenode.http-address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop102:9870<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- secondary namenode web访问地址 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.namenode.secondary.http-address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop104:9868<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- 数据的副本数量 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.replication<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>2<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h3 id="mapred-site-xml"><a href="#mapred-site-xml" class="headerlink" title="mapred-site.xml"></a>mapred-site.xml</h3><figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.framework.name<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>yarn<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- 历史服务器端地址 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop104:10020<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- 历史服务器web端地址 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.webapp.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop104:19888<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h3 id="yarn-site-xml"><a href="#yarn-site-xml" class="headerlink" title="yarn-site.xml"></a>yarn-site.xml</h3><figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">&lt;!-- Site specific YARN configuration properties --&gt;</span></span><br><span class="line"><span class="comment">&lt;!-- Reducer获取数据的方式 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.aux-services<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>mapreduce_shuffle<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- 指定YARN的ResourceManager的地址 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.resourcemanager.hostname<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop103<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- 日志聚集功能使能 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.log-aggregation-enable<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>true<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- 日志保留时间设置7天 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.log-aggregation.retain-seconds<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>604800<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">&lt;!-- 环境变量的继承 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.env-whitelist<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h3 id="workers"><a href="#workers" class="headerlink" title="workers"></a>workers</h3><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">hadoop102</span><br><span class="line">hadoop103</span><br><span class="line">hadoop104</span><br></pre></td></tr></table></figure>

<p>注意：该文件中，添加配置时，内容结尾不允许有空格，或空行。</p>
<p>将配置同步到群集其他节点</p>
<h2 id="三、启动集群"><a href="#三、启动集群" class="headerlink" title="三、启动集群"></a>三、启动集群</h2><h3 id="格式化-namenode"><a href="#格式化-namenode" class="headerlink" title="格式化 namenode"></a>格式化 namenode</h3><p>如果集群是第一次启动，需要在启动前对 namenode 进行格式化</p>
<blockquote>
<p>注意：格式化 namenode 后，会产生新的集群id，导致 namenode 和 datanode 的集群id不一致，从而集群找不到数据。</p>
<p>如果集群运行中报错，需要重新对 namenode 进行格式化的话，一定要先停止 namenode 和 datanode 进程，并且需要删除所有机器的 data 和 logs 目录，然后再进行格式化。</p>
</blockquote>
<p>在 hadoop102 节点（namenode 所在节点）格式化 namenode</p>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./bin/hadoop namenode -format</span><br></pre></td></tr></table></figure>

<p>格式化后，如果输出 “successfully formatted.” 信息，即格式化成功</p>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">INFO common.Storage: Storage directory ... ... has been successfully formatted.</span><br></pre></td></tr></table></figure>

<h3 id="启停-HDFS"><a href="#启停-HDFS" class="headerlink" title="启停 HDFS"></a>启停 HDFS</h3><p>HDFS 启动的进程为 namenode 和 datanode，直接在部署了 namenode 节点（hadoop102）执行以下命令：</p>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">./sbin/start-dfs.sh</span><br><span class="line">./sbin/stop-dfs.sh</span><br></pre></td></tr></table></figure>

<p>单节点启停</p>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">./sbin/hdfs --daemon [start|stop] datanode</span><br><span class="line">./sbin/hdfs --daemon [start|stop] namenode</span><br></pre></td></tr></table></figure>

<p>在 web 页面查看 HDFS 的 namenode</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">http://hadoop102:9870</span><br></pre></td></tr></table></figure>

<h3 id="启停-YARN"><a href="#启停-YARN" class="headerlink" title="启停 YARN"></a>启停 YARN</h3><p>YARN 启动的进程为 resourcemanager 和 nodemanager，直接在部署了 resourcemanager 节点（hadoop103）执行以下命令：</p>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">./sbin/start-yarn.sh</span><br><span class="line">./sbin/stop-yarn.sh</span><br></pre></td></tr></table></figure>

<p>单节点启停</p>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">./sbin/yarn --daemon [start|stop] nodemanager</span><br><span class="line">./sbin/yarn --daemon [start|stop] resourcemanager</span><br></pre></td></tr></table></figure>

<p>在 web 页面查看 YARN 的 resourcemanager</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">http://hadoop103:8088</span><br></pre></td></tr></table></figure>

<h3 id="启动历史服务器"><a href="#启动历史服务器" class="headerlink" title="启动历史服务器"></a>启动历史服务器</h3><p>启动历史服务器，直接在部署了 historyserver 节点执行以下命令</p>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./bin/mapred --daemon [start|stop] historyserver</span><br></pre></td></tr></table></figure>

<p>在 web 页面查看 historyserver</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">http://hadoop104:19888</span><br></pre></td></tr></table></figure>

<h3 id="所有进程"><a href="#所有进程" class="headerlink" title="所有进程"></a>所有进程</h3><figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">===== ===== hadoop102 ===== =====</span><br><span class="line">5025 NodeManager</span><br><span class="line">4342 NameNode</span><br><span class="line">4505 DataNode</span><br><span class="line">5487 Jps</span><br><span class="line">===== ===== hadoop103 ===== =====</span><br><span class="line">4258 ResourceManager</span><br><span class="line">3829 DataNode</span><br><span class="line">3997 NodeManager</span><br><span class="line">4814 Jps</span><br><span class="line">===== ===== hadoop104 ===== =====</span><br><span class="line">3745 SecondaryNameNode</span><br><span class="line">3875 NodeManager</span><br><span class="line">4195 JobHistoryServer</span><br><span class="line">3623 DataNode</span><br><span class="line">4407 Jps</span><br></pre></td></tr></table></figure>

<h2 id="四、测试集群"><a href="#四、测试集群" class="headerlink" title="四、测试集群"></a>四、测试集群</h2><p>启动集群后，通过 hadoop 自带的 wordcount 示例程序，测试 hadoop 集群搭建是否成功</p>
<ol>
<li>创建测试文件</li>
</ol>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">hello hadoop</span><br><span class="line">hadoop hdfs</span><br><span class="line">yarn</span><br><span class="line">hadoop</span><br><span class="line">hello</span><br></pre></td></tr></table></figure>

<ol start="2">
<li>创建目录，并上传测试文件到 HDFS</li>
</ol>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#在hdfs上，创建指定目录</span></span><br><span class="line">hadoop fs -<span class="built_in">mkdir</span> -p /test/input</span><br><span class="line"><span class="comment">#上传测试文件到hdfs</span></span><br><span class="line">hadoop fs -put input.txt /test/input</span><br></pre></td></tr></table></figure>

<ol start="3">
<li>运行 wordcount 示例程序</li>
</ol>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-3.1.3.jar wordcount /test/input /test/output</span><br></pre></td></tr></table></figure>

<ol start="4">
<li>查看输出</li>
</ol>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -<span class="built_in">cat</span> /test/output/*</span><br></pre></td></tr></table></figure>

<p>输出结果：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">hadoop	3</span><br><span class="line">hdfs	1</span><br><span class="line">hello	2</span><br><span class="line">yarn	1</span><br></pre></td></tr></table></figure>

<h2 id="五、其他配置"><a href="#五、其他配置" class="headerlink" title="五、其他配置"></a>五、其他配置</h2><h3 id="历史服务器"><a href="#历史服务器" class="headerlink" title="历史服务器"></a>历史服务器</h3><p>历史服务器，可以查看程序的历史运行情况</p>
<h3 id="日志聚集功能"><a href="#日志聚集功能" class="headerlink" title="日志聚集功能"></a>日志聚集功能</h3><p>应用运行完成后，会将程序运行日志上传到 HDFS 系统上。</p>
<h1 id="集群启停脚本"><a href="#集群启停脚本" class="headerlink" title="集群启停脚本"></a>集群启停脚本</h1><figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#! /bin/bash</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#1. 判断参数个数</span></span><br><span class="line"><span class="keyword">if</span> [ <span class="variable">$#</span> -lt 1 ]</span><br><span class="line"><span class="keyword">then</span></span><br><span class="line">    <span class="built_in">echo</span> Not Enough Arguement!</span><br><span class="line">    <span class="built_in">exit</span>;</span><br><span class="line"><span class="keyword">fi</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#2. 参数不为空，则执行对应命令</span></span><br><span class="line"><span class="keyword">case</span> <span class="variable">$1</span> <span class="keyword">in</span></span><br><span class="line"><span class="string">&quot;start&quot;</span>)&#123;</span><br><span class="line">	ssh hadoop102 /opt/module/hadoop-3.1.3/sbin/start-dfs.sh</span><br><span class="line">	ssh hadoop103 /opt/module/hadoop-3.1.3/sbin/start-yarn.sh</span><br><span class="line">	ssh hadoop104 /opt/module/hadoop-3.1.3/sbin/mr-jobhistory-daemon.sh start historyserver</span><br><span class="line">&#125;;;</span><br><span class="line"></span><br><span class="line"><span class="string">&quot;stop&quot;</span>)&#123;</span><br><span class="line">	ssh hadoop102 /opt/module/hadoop-3.1.3/sbin/stop-dfs.sh</span><br><span class="line">	ssh hadoop103 /opt/module/hadoop-3.1.3/sbin/stop-yarn.sh</span><br><span class="line">	ssh hadoop104 /opt/module/hadoop-3.1.3/sbin/mr-jobhistory-daemon.sh stop historyserver</span><br><span class="line">&#125;;;</span><br><span class="line"></span><br><span class="line"><span class="keyword">esac</span></span><br></pre></td></tr></table></figure>

<h1 id="常用端口号汇总"><a href="#常用端口号汇总" class="headerlink" title="常用端口号汇总"></a>常用端口号汇总</h1><table>
<thead>
<tr>
<th>端口名称</th>
<th>端口</th>
<th>配置文件</th>
<th>配置</th>
</tr>
</thead>
<tbody><tr>
<td>NameNode内部通信端口</td>
<td>8020</td>
<td>core-site.xml</td>
<td>fs.defaultFS</td>
</tr>
<tr>
<td>NameNode http端口</td>
<td>9870</td>
<td>hdfs-site.xml</td>
<td>dfs.namenode.http-address</td>
</tr>
<tr>
<td>SecondaryNameNode http端口</td>
<td>9868</td>
<td>hdfs-site.xml</td>
<td>dfs.namenode.secondary.http-address</td>
</tr>
<tr>
<td>YARN resourcemanager http端口</td>
<td>8088</td>
<td>yarn-site.xml</td>
<td>yarn.resourcemanager.webapp.address</td>
</tr>
<tr>
<td>历史服务器通信端口</td>
<td>19888</td>
<td>mapred-site.xml</td>
<td>mapreduce.jobhistory.webapp.address</td>
</tr>
</tbody></table>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="mailto:undefined">Oct25-X</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="https://oct25-x.atomgit.net/oct25-xxxxx/2023/05/29/03%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop02_%E9%9B%86%E7%BE%A4%E6%90%AD%E5%BB%BA/">https://oct25-x.atomgit.net/oct25-xxxxx/2023/05/29/03%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop02_%E9%9B%86%E7%BE%A4%E6%90%AD%E5%BB%BA/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://oct25-x.atomgit.net/oct25-xxxxx" target="_blank">Oct25-X</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/oct25-xxxxx/tags/Hadoop/">Hadoop</a></div><div class="post_share"><div class="social-share" data-image="/oct25-xxxxx/img/cover_img/ctt3.png" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/oct25-xxxxx/2023/06/02/04%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop03_%E5%88%86%E5%B8%83%E5%BC%8F%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%20HDFS/"><img class="prev-cover" src="/oct25-xxxxx/img/cover_img/ctt4.png" onerror="onerror=null;src='/oct25-xxxxx/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">大数据之Hadoop03_分布式文件系统 HDFS</div></div></a></div><div class="next-post pull-right"><a href="/oct25-xxxxx/2023/05/28/02%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop01_%E5%85%A5%E9%97%A8/"><img class="next-cover" src="/oct25-xxxxx/img/cover_img/ctt2.png" onerror="onerror=null;src='/oct25-xxxxx/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">大数据之Hadoop01_入门</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span>相关推荐</span></div><div class="relatedPosts-list"><div><a href="/oct25-xxxxx/2023/05/28/02%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop01_%E5%85%A5%E9%97%A8/" title="大数据之Hadoop01_入门"><img class="cover" src="/oct25-xxxxx/img/cover_img/ctt2.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-05-28</div><div class="title">大数据之Hadoop01_入门</div></div></a></div><div><a href="/oct25-xxxxx/2023/06/02/04%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop03_%E5%88%86%E5%B8%83%E5%BC%8F%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%20HDFS/" title="大数据之Hadoop03_分布式文件系统 HDFS"><img class="cover" src="/oct25-xxxxx/img/cover_img/ctt4.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-06-02</div><div class="title">大数据之Hadoop03_分布式文件系统 HDFS</div></div></a></div><div><a href="/oct25-xxxxx/2023/06/06/05%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop04_%E5%88%86%E5%B8%83%E5%BC%8F%E8%AE%A1%E7%AE%97%E6%A1%86%E6%9E%B6%20MapReduce/" title="大数据之Hadoop04_分布式计算框架 MapReduce"><img class="cover" src="/oct25-xxxxx/img/cover_img/ctt5.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-06-06</div><div class="title">大数据之Hadoop04_分布式计算框架 MapReduce</div></div></a></div><div><a href="/oct25-xxxxx/2023/06/15/06%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop05_%E5%88%86%E5%B8%83%E5%BC%8F%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6%E6%A1%86%E6%9E%B6%20Yarn/" title="大数据之Hadoop05_分布式资源调度框架 Yarn"><img class="cover" src="/oct25-xxxxx/img/cover_img/ctt6.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-06-15</div><div class="title">大数据之Hadoop05_分布式资源调度框架 Yarn</div></div></a></div><div><a href="/oct25-xxxxx/2023/06/17/07%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop06_%E7%B3%BB%E7%BB%9F%E8%B0%83%E4%BC%98%E6%89%8B%E5%86%8C/" title="大数据之Hadoop06_系统调优手册"><img class="cover" src="/oct25-xxxxx/img/cover_img/ctt7.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-06-17</div><div class="title">大数据之Hadoop06_系统调优手册</div></div></a></div><div><a href="/oct25-xxxxx/2023/06/23/08%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8BHadoop07_Hadoop%20%E9%AB%98%E5%8F%AF%E7%94%A8%E9%83%A8%E7%BD%B2/" title="大数据之Hadoop07_Hadoop 高可用部署"><img class="cover" src="/oct25-xxxxx/img/cover_img/ctt8.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-06-23</div><div class="title">大数据之Hadoop07_Hadoop 高可用部署</div></div></a></div></div></div></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="/oct25-xxxxx/img/avatar.jpg" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">Oct25-X</div><div class="author-info__description"></div></div><div class="card-info-data is-center"><div class="card-info-data-item"><a href="/oct25-xxxxx/archives/"><div class="headline">文章</div><div class="length-num">52</div></a></div><div class="card-info-data-item"><a href="/oct25-xxxxx/tags/"><div class="headline">标签</div><div class="length-num">17</div></a></div><div class="card-info-data-item"><a href="/oct25-xxxxx/categories/"><div class="headline">分类</div><div class="length-num">8</div></a></div></div><a id="card-info-btn" target="_blank" rel="noopener" href="https://github.com/xxxxxx"><i class="fab fa-github"></i><span>Follow Me</span></a></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E9%9B%86%E7%BE%A4%E8%A7%84%E5%88%92"><span class="toc-number">1.</span> <span class="toc-text">集群规划</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#JDK-%E5%AE%89%E8%A3%85"><span class="toc-number">2.</span> <span class="toc-text">JDK 安装</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E9%9B%86%E7%BE%A4%E9%97%B4%E5%85%8D%E5%AF%86%E7%99%BB%E5%BD%95"><span class="toc-number">3.</span> <span class="toc-text">集群间免密登录</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#Hadoop-%E9%9B%86%E7%BE%A4%E6%90%AD%E5%BB%BA"><span class="toc-number">4.</span> <span class="toc-text">Hadoop 集群搭建</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%80%E3%80%81%E5%AE%89%E8%A3%85-Hadoop"><span class="toc-number">4.1.</span> <span class="toc-text">一、安装 Hadoop</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BA%8C%E3%80%81%E4%BF%AE%E6%94%B9%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6"><span class="toc-number">4.2.</span> <span class="toc-text">二、修改配置文件</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#core-site-xml"><span class="toc-number">4.2.1.</span> <span class="toc-text">core-site.xml</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#hdfs-site-xml"><span class="toc-number">4.2.2.</span> <span class="toc-text">hdfs-site.xml</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#mapred-site-xml"><span class="toc-number">4.2.3.</span> <span class="toc-text">mapred-site.xml</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#yarn-site-xml"><span class="toc-number">4.2.4.</span> <span class="toc-text">yarn-site.xml</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#workers"><span class="toc-number">4.2.5.</span> <span class="toc-text">workers</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%89%E3%80%81%E5%90%AF%E5%8A%A8%E9%9B%86%E7%BE%A4"><span class="toc-number">4.3.</span> <span class="toc-text">三、启动集群</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%A0%BC%E5%BC%8F%E5%8C%96-namenode"><span class="toc-number">4.3.1.</span> <span class="toc-text">格式化 namenode</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%90%AF%E5%81%9C-HDFS"><span class="toc-number">4.3.2.</span> <span class="toc-text">启停 HDFS</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%90%AF%E5%81%9C-YARN"><span class="toc-number">4.3.3.</span> <span class="toc-text">启停 YARN</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%90%AF%E5%8A%A8%E5%8E%86%E5%8F%B2%E6%9C%8D%E5%8A%A1%E5%99%A8"><span class="toc-number">4.3.4.</span> <span class="toc-text">启动历史服务器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%89%80%E6%9C%89%E8%BF%9B%E7%A8%8B"><span class="toc-number">4.3.5.</span> <span class="toc-text">所有进程</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%9B%9B%E3%80%81%E6%B5%8B%E8%AF%95%E9%9B%86%E7%BE%A4"><span class="toc-number">4.4.</span> <span class="toc-text">四、测试集群</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BA%94%E3%80%81%E5%85%B6%E4%BB%96%E9%85%8D%E7%BD%AE"><span class="toc-number">4.5.</span> <span class="toc-text">五、其他配置</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%8E%86%E5%8F%B2%E6%9C%8D%E5%8A%A1%E5%99%A8"><span class="toc-number">4.5.1.</span> <span class="toc-text">历史服务器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%97%A5%E5%BF%97%E8%81%9A%E9%9B%86%E5%8A%9F%E8%83%BD"><span class="toc-number">4.5.2.</span> <span class="toc-text">日志聚集功能</span></a></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E9%9B%86%E7%BE%A4%E5%90%AF%E5%81%9C%E8%84%9A%E6%9C%AC"><span class="toc-number">5.</span> <span class="toc-text">集群启停脚本</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%B8%B8%E7%94%A8%E7%AB%AF%E5%8F%A3%E5%8F%B7%E6%B1%87%E6%80%BB"><span class="toc-number">6.</span> <span class="toc-text">常用端口号汇总</span></a></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/oct25-xxxxx/2024/02/24/52%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala09_%E9%9A%90%E5%BC%8F%E8%BD%AC%E6%8D%A2%E3%80%81%E5%BC%82%E5%B8%B8%E5%92%8C%E6%B3%9B%E5%9E%8B/" title="大数据之多范式编程语言 Scala09_隐式转换、异常和泛型"><img src="/oct25-xxxxx/img/cover_img/ctt52.png" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/404.jpg'" alt="大数据之多范式编程语言 Scala09_隐式转换、异常和泛型"/></a><div class="content"><a class="title" href="/oct25-xxxxx/2024/02/24/52%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala09_%E9%9A%90%E5%BC%8F%E8%BD%AC%E6%8D%A2%E3%80%81%E5%BC%82%E5%B8%B8%E5%92%8C%E6%B3%9B%E5%9E%8B/" title="大数据之多范式编程语言 Scala09_隐式转换、异常和泛型">大数据之多范式编程语言 Scala09_隐式转换、异常和泛型</a><time datetime="2024-02-24T04:55:34.000Z" title="发表于 2024-02-24 12:55:34">2024-02-24</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/oct25-xxxxx/2024/02/24/51%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala08_%E6%A8%A1%E5%BC%8F%E5%8C%B9%E9%85%8D/" title="大数据之多范式编程语言 Scala08_模式匹配"><img src="/oct25-xxxxx/img/cover_img/ctt51.png" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/404.jpg'" alt="大数据之多范式编程语言 Scala08_模式匹配"/></a><div class="content"><a class="title" href="/oct25-xxxxx/2024/02/24/51%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala08_%E6%A8%A1%E5%BC%8F%E5%8C%B9%E9%85%8D/" title="大数据之多范式编程语言 Scala08_模式匹配">大数据之多范式编程语言 Scala08_模式匹配</a><time datetime="2024-02-24T00:46:12.000Z" title="发表于 2024-02-24 08:46:12">2024-02-24</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/oct25-xxxxx/2024/02/22/50%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala07_%E9%9B%86%E5%90%88/" title="大数据之多范式编程语言 Scala07_集合"><img src="/oct25-xxxxx/img/cover_img/ctt50.png" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/404.jpg'" alt="大数据之多范式编程语言 Scala07_集合"/></a><div class="content"><a class="title" href="/oct25-xxxxx/2024/02/22/50%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala07_%E9%9B%86%E5%90%88/" title="大数据之多范式编程语言 Scala07_集合">大数据之多范式编程语言 Scala07_集合</a><time datetime="2024-02-22T08:16:10.000Z" title="发表于 2024-02-22 16:16:10">2024-02-22</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/oct25-xxxxx/2024/02/21/49%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala06_%E9%9D%A2%E5%90%91%E5%AF%B9%E8%B1%A1/" title="大数据之多范式编程语言 Scala06_面向对象"><img src="/oct25-xxxxx/img/cover_img/ctt49.png" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/404.jpg'" alt="大数据之多范式编程语言 Scala06_面向对象"/></a><div class="content"><a class="title" href="/oct25-xxxxx/2024/02/21/49%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala06_%E9%9D%A2%E5%90%91%E5%AF%B9%E8%B1%A1/" title="大数据之多范式编程语言 Scala06_面向对象">大数据之多范式编程语言 Scala06_面向对象</a><time datetime="2024-02-21T07:01:10.000Z" title="发表于 2024-02-21 15:01:10">2024-02-21</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/oct25-xxxxx/2024/02/20/48%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala05_%E5%87%BD%E6%95%B0%E5%BC%8F%E7%BC%96%E7%A8%8B/" title="大数据之多范式编程语言 Scala05_函数式编程"><img src="/oct25-xxxxx/img/cover_img/ctt48.png" onerror="this.onerror=null;this.src='/oct25-xxxxx/img/404.jpg'" alt="大数据之多范式编程语言 Scala05_函数式编程"/></a><div class="content"><a class="title" href="/oct25-xxxxx/2024/02/20/48%E5%A4%A7%E6%95%B0%E6%8D%AE%E4%B9%8B%E5%A4%9A%E8%8C%83%E5%BC%8F%E7%BC%96%E7%A8%8B%E8%AF%AD%E8%A8%80Scala05_%E5%87%BD%E6%95%B0%E5%BC%8F%E7%BC%96%E7%A8%8B/" title="大数据之多范式编程语言 Scala05_函数式编程">大数据之多范式编程语言 Scala05_函数式编程</a><time datetime="2024-02-20T12:05:11.000Z" title="发表于 2024-02-20 20:05:11">2024-02-20</time></div></div></div></div></div></div></main><footer id="footer" style="background-image: url('/oct25-xxxxx/img/top_img/top_bg2.jpg')"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2024 By Oct25-X</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="translateLink" type="button" title="简繁转换">繁</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><nav class="search-nav"><span class="search-dialog-title">本地搜索</span><span id="loading-status"></span><button class="search-close-button"><i class="fas fa-times"></i></button></nav><div class="is-center" id="loading-database"><i class="fas fa-spinner fa-pulse"></i><span>  数据库加载中</span></div><div class="search-wrap"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div><hr/><div id="local-search-results"></div></div></div><div id="search-mask"></div></div><div id="rightMenu"><div class="rightMenu-group rightMenu-small"><div class="rightMenu-item" id="menu-backward"><i class="fa-solid fa-arrow-left"></i></div><div class="rightMenu-item" id="menu-forward"><i class="fa-solid fa-arrow-right"></i></div><div class="rightMenu-item" id="menu-refresh"><i class="fa-solid fa-arrow-rotate-right"></i></div><div class="rightMenu-item" id="menu-home"><i class="fa-solid fa-house"></i></div></div><div class="rightMenu-group rightMenu-line rightMenuOther"><a class="rightMenu-item menu-link" href="/archives/"><i class="fa-solid fa-archive"></i><span>文章归档</span></a><a class="rightMenu-item menu-link" href="/categories/"><i class="fa-solid fa-folder-open"></i><span>文章分类</span></a><a class="rightMenu-item menu-link" href="/tags/"><i class="fa-solid fa-tags"></i><span>文章标签</span></a></div><div class="rightMenu-group rightMenu-line rightMenuNormal"><a class="rightMenu-item" id="menu-translate"><i class="fa-solid fa-earth-asia"></i><span>繁简切换</span></a><div class="rightMenu-item" id="menu-darkmode"><i class="fa-solid fa-moon"></i><span>切换模式</span></div></div></div><div id="rightmenu-mask"></div><div><script src="/oct25-xxxxx/js/utils.js"></script><script src="/oct25-xxxxx/js/main.js"></script><script src="/oct25-xxxxx/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.umd.js"></script><script src="/oct25-xxxxx/js/search/local-search.js"></script><div class="js-pjax"></div><script defer src="https://npm.elemecdn.com/jquery@latest/dist/jquery.min.js"></script><script defer data-pjax src="/js/rightMenu.js"></script><script defer="defer" id="ribbon" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/canvas-ribbon.min.js" size="150" alpha="0.6" zIndex="-1" mobile="false" data-click="true"></script><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/activate-power-mode.min.js"></script><script>POWERMODE.colorful = true;
POWERMODE.shake = true;
POWERMODE.mobile = false;
document.body.addEventListener('input', POWERMODE);
</script><script id="click-heart" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/click-heart.min.js" async="async" mobile="false"></script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>